﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Web;
using ICSharpCode.SharpZipLib.Zip;
using SDT.CodeSystem;

using SDT.CrawlSystem.Servers.Filters;
using SDT.FileSystem;
using SDT.Libraries.Convertor;
using SDT.ServiceSystem;
using SDT.DataService.Services;
using System.Text.RegularExpressions;
using SDT.Libraries.DuplicateDetection;
using SDT.DataService;
using SDT.Libraries.Utilities;
using SDT.CrawlSystem.Servers.Save;

namespace SDT.CrawlSystem.Servers
{

    public sealed partial class WebCrawler : WorkerThread
    {
        private const string XML_RESULT_TEMPLATE =
            "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\r\n<_{0}>\r\n\t<TF><![CDATA[{1}]]></TF>\r\n\t <LF><![CDATA[{7}]]></LF>\r\n\t<CF><![CDATA[{2}]]></CF>\r\n\t<PF>{3}</PF>\r\n\t<MF>{4}</MF>\r\n\t<TXT><![CDATA[{5}]]></TXT>\r\n</_{6}>";

        private WCContainer TheContainer = null;
        private LinkDownloadService svDownload = new LinkDownloadService();
        private PageService svPage = new PageService();
        private NewsService svNews = new NewsService();
        private ErrorNewsService svErrorNews = new ErrorNewsService();

        #region Properties from data file
        private string dataFile = null;
        public string DataFile
        {
            get { return dataFile; }
        }

        private int interval = 10000;
        public override int Interval
        {
            get { return interval; }
        }

        private int workUnitInterval = 1500;
        public override int WorkUnitInterval
        {
            get { return workUnitInterval; }
        }
        
        #endregion

        #region Constructors
        internal WebCrawler(string dataFile, WCContainer container)
            : base()
        {
            TheContainer = container;
            this.dataFile = dataFile;

            LoadData();
        }
        #endregion

        protected override bool CanStop
        {
            get
            {
                lock (UrlLoaders)
                {
                    foreach (var item in UrlLoaders)
                        if (!item.EOF)
                            return false;
                    return true;
                }
            }
        }

        private void RefreshUrlLoaders()
        {
            lock (Logger)
                Logger.AddEntry("BEGIN RefreshUrlLoaders");
            lock (UrlLoaders)
                foreach (var item in UrlLoaders)
                    item.Refresh();
            lock (Logger)
            {
                Logger.AddEntry("END RefreshUrlLoaders");
                Logger.Flush();
            }
        }

        private void RetrieveWorkUnits()
        {
            lock (Logger)
                Logger.AddEntry("BEGIN RetrieveWorkUnits");
            lock (UrlLoaders)
            {
                foreach (var item in UrlLoaders)
                {
                    if (!item.EOF)
                    {
                        lock (Logger)
                            Logger.AddEntry(String.Format("PID = {0} - NPages = {1} - PageNo = {2}", item.PID, item.TotalPages, item.CurrentPage), 1);
                        var l = item.RetrieveWorkUnits();
                        foreach (var w in l)
                            AddWork(w);
                        lock (Logger)
                            Logger.AddEntry(String.Format("PID = {0} - NUrls = {1}", item.PID, item.TotalUrls), 1);
                    }
                }
            }
            lock (Logger)
            {
                Logger.AddEntry("END RetrieveWorkUnits");
                Logger.Flush();
            }
        }

        public override void DoWork()
        {
            try
            {
                #region Load work units
                // Prepare work unit list
                RefreshUrlLoaders();
                // Retrieve work unit list
                RetrieveWorkUnits();
                #endregion

                while (true)
                {
                    // Load data from xml file (if necessary)
                    LoadData();

                    if (!HaveWork)
                    {
                        RefreshUrlLoaders();
                        RetrieveWorkUnits();
                    }

                    while (!CanStop || HaveWork)
                    {
                        while (HaveWork)
                        {
                            // Get work item
                            var w = GetWork();
                            if (w != null && w.Ready)
                                DownloadWebsite(w as WebWorkUnit);

                            Logger.Flush();

                            Thread.Sleep(WorkUnitInterval);
                            //break;
                        }

                        // Add next work units
                        RetrieveWorkUnits();

                        Thread.Sleep(WorkUnitInterval);
                        //break;
                    }

                    Thread.Sleep(Interval);
                }
            }
            catch (Exception ex)
            {
                lock (Logger)
                {
                    Logger.AddEntry(string.Format("EXCEPTION {0} [{1}]", ex, ex.StackTrace));
                    Logger.Flush();
                }
            }
        }

        #region Download webpage
        private class RequestState
        {
            // Holds the request
            public WebRequest Request;

            public WebWorkUnit WorkUnit;

            public RequestState(WebRequest request, WebWorkUnit workUnit)
            {
                Request = request;
                WorkUnit = workUnit;
            }
        }

        private CookieContainer cookie = null;
        public CookieContainer Cookie
        {
            get
            {
                if (cookie == null)
                    cookie = new CookieContainer();
                return cookie;
            }
        }

        private void DownloadWebsite(WebWorkUnit w)
        {
            lock (Logger)
                Logger.AddEntry(String.Format("GET: UID = {0} - {1}", w.Uid, w.Url));
            lock (w)
            {
                w.Ready = false;
                w.RawHtml = null;
            }

            try
            {
                var r = (HttpWebRequest)HttpWebRequest.Create(w.Url);
                r.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.8) Gecko/20100202 Firefox/3.5.8 GTB6 (.NET CLR 3.5.30729)";
                //r.UserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)";

                // Maintain same cookie
                r.CookieContainer = Cookie;
                r.Method = "GET";
                //Encode = w.TheUFilter.Enc;
                var state = new RequestState(r, w);

                var result = r.BeginGetResponse(new AsyncCallback(DownloadCompletedCallback), state);

                // Register the timeout callback
                ThreadPool.RegisterWaitForSingleObject(
                    result.AsyncWaitHandle, new WaitOrTimerCallback(DownloadTimeoutCallback), state, 30 * 1000, true);
            }
            catch (Exception ex)
            {
                w.ChangeStatus(UrlLoader.Status.TIMEOUT);
                lock (Logger)
                    Logger.AddEntry(String.Format("EXC: UID = {0} - {1} = {2}", w.Uid, ex, ex.Message));
            }
        }

        private void DownloadCompletedCallback(IAsyncResult result)
        {
            var state = result.AsyncState as RequestState;
            try
            {
                var req = state.Request as WebRequest;

                #region Get the Response
                var res = req.EndGetResponse(result) as HttpWebResponse;
                var s = res.GetResponseStream() as Stream;
                var r = new StreamReader(s, state.WorkUnit.TheUFilter.Enc);
                //var r = new StreamReader(s);
                lock (state.WorkUnit)
                {
                    state.WorkUnit.RawHtml = r.ReadToEnd();
                    state.WorkUnit.Ready = true;
                    if (ResultHandler != null)
                        ResultHandler(state.WorkUnit);
                    lock (Logger)
                        Logger.AddEntry(String.Format("SUC: UID = {0}", (result.AsyncState as RequestState).WorkUnit.Uid));
                }
                #endregion

                #region Release resources
                res.Close();
                s.Close();
                r.Close();
                #endregion
            }
            #region WebException
            catch (WebException ex)
            {
                switch (ex.Status)
                {
                    case WebExceptionStatus.CacheEntryNotFound:
                        break;
                    case WebExceptionStatus.ConnectFailure:
                        break;
                    case WebExceptionStatus.ConnectionClosed:
                        break;
                    case WebExceptionStatus.KeepAliveFailure:
                        break;
                    case WebExceptionStatus.MessageLengthLimitExceeded:
                        break;
                    case WebExceptionStatus.NameResolutionFailure:
                        state.WorkUnit.ChangeStatus(UrlLoader.Status.OFFLINE);
                        break;
                    case WebExceptionStatus.Pending:
                        break;
                    case WebExceptionStatus.PipelineFailure:
                        break;
                    case WebExceptionStatus.ProtocolError:
                        state.WorkUnit.ChangeStatus(UrlLoader.Status.NOT_FOUND);
                        break;
                    case WebExceptionStatus.ProxyNameResolutionFailure:
                        break;
                    case WebExceptionStatus.ReceiveFailure:
                        break;
                    case WebExceptionStatus.RequestCanceled:
                        break;
                    case WebExceptionStatus.RequestProhibitedByCachePolicy:
                        break;
                    case WebExceptionStatus.RequestProhibitedByProxy:
                        break;
                    case WebExceptionStatus.SecureChannelFailure:
                        break;
                    case WebExceptionStatus.SendFailure:
                        break;
                    case WebExceptionStatus.ServerProtocolViolation:
                        break;
                    case WebExceptionStatus.Success:
                        break;
                    case WebExceptionStatus.Timeout:
                        break;
                    case WebExceptionStatus.TrustFailure:
                        break;
                    case WebExceptionStatus.UnknownError:
                        break;
                    default:
                        state.WorkUnit.ChangeStatus(UrlLoader.Status.INVALID);
                        break;
                };

                lock (Logger)
                    Logger.AddEntry(String.Format("EXC: UID = {0} - {1}", state.WorkUnit.Uid, ex.Status));
            }
            #endregion
            #region Other exceptions
            catch (Exception ex)
            {
                state.WorkUnit.ChangeStatus(UrlLoader.Status.INVALID);
                lock (Logger)
                    Logger.AddEntry(String.Format("EXC: UID = {0} - {1} = {2}", state.WorkUnit.Uid, ex, ex.Message));
            }
            #endregion
        }

        private void DownloadTimeoutCallback(object state, bool timedOut)
        {
            if (timedOut)
            {
                var r = state as RequestState;
                if (r != null)
                    r.Request.Abort();

                lock (r.WorkUnit)
                {
                    r.WorkUnit.ChangeStatus(UrlLoader.Status.TIMEOUT);
                    r.WorkUnit.ClearResult();
                    lock (Logger)
                        Logger.AddEntry(String.Format("TIMEOUT @ UID = {0}", r.WorkUnit.Uid));
                }
            }
        }
        #endregion

        private static string ConcatinateList(IList<string> l)
        {
            if (l == null || l.Count == 0)
                return "";
            var b = new StringBuilder();
            foreach (var i in l)
                b.Append(i).Append(';');
            return b.ToString();
        }

        #region Result handler
          private bool SaveResult(IWorkUnit w)
        {
            if (w == null || !w.Ready || (w as WebWorkUnit).RawHtml == null)
                return false;

            var _w = w as WebWorkUnit;

            string txt = null;
            var hf = _w.TheUFilter.HFilter.DoFilter(_w.RawHtml, txt, ref txt, true);

            txt = HttpUtility.HtmlDecode(txt);

            #region Do title filter
            string tf = null;
            var temp = _w.TheUFilter.TFilter.DoFilter(_w.TheUFilter.TFilter.DataSource == Filter.Regions.WEB? _w.RawHtml : hf, txt, ref tf, true);
            if (temp != null)
                tf = temp;
            if (_w.TheUFilter.Enc.Equals(Encoding.GetEncoding("iso-8859-1")))
                tf = HttpUtility.HtmlDecode(iso8859ToUnicode(tf));
            else
                tf = HttpUtility.HtmlDecode(tf);
            tf = ConvertNewlinesToSingleSpaces(StripHTML(tf));
            tf = tf.Trim();         // Added by VietLV (Add date 01/06/2011)        

            #endregion

            #region Do lead filter
            string lf = null;
            temp = _w.TheUFilter.LFilter.DoFilter(_w.TheUFilter.LFilter.DataSource == Filter.Regions.WEB ? _w.RawHtml : hf, txt, ref lf, true);
            if (temp != null)
                lf = temp;

            if (_w.TheUFilter.Enc.Equals(Encoding.GetEncoding("iso-8859-1")))
                lf = HttpUtility.HtmlDecode(HttpUtility.HtmlEncode(iso8859ToUnicode(lf)));
            else
                lf = HttpUtility.HtmlDecode(lf);
            //lf = ConvertNewlinesToSingleSpaces(StripHTML(lf));
            lf =lf.Trim();             // Added by VietLV (Add date 01/06/2011)
            #endregion

            #region Date time
            var df = _w.TheUFilter.DFilter.DoFilter(_w.TheUFilter.DFilter.DataSource == Filter.Regions.WEB ? _w.RawHtml : hf, txt);
            #endregion


            #region Do content filter
            string cf = null;
            temp = null;

            temp = _w.TheUFilter.ConFilter.DoFilter(_w.TheUFilter.ConFilter.DataSource == Filter.Regions.WEB ? _w.RawHtml : hf, txt, ref cf, true);
            if (temp != null)
                cf = temp;

            string contentRawHtml = null;
            contentRawHtml = cf;
              //remove div first
            cf = cf.Replace("IMG", "img");
            cf = cf.Replace("DIV", "div");
            cf = CleanHtml(cf);
            // remove other div  VietLV
            
            cf = StripTagsAndAttributes(cf, _w.TheUFilter.AllowTag);
            if (_w.TheUFilter.Enc.Equals(Encoding.GetEncoding("iso-8859-1")))
                cf = HttpUtility.HtmlDecode(HttpUtility.HtmlEncode(iso8859ToUnicode(cf)));
            if (lf.Equals(""))
            {
                List<string> strResult = LeadFilter.ProcessLead(cf);
                if (strResult.Count > 0)
                {
                    if (_w.TheUFilter.Enc.Equals(Encoding.GetEncoding("iso-8859-1")))
                    {
                        lf = HttpUtility.HtmlDecode(HttpUtility.HtmlEncode(iso8859ToUnicode(strResult[0])));

                        cf = HttpUtility.HtmlDecode(HttpUtility.HtmlEncode(iso8859ToUnicode(strResult[1])));
                    }
                    else
                    {
                        lf = strResult[0];
                        cf = strResult[1];
                    }
                }
            }
            else
            {
                
                cf = cf.Replace(lf, "");  // Disable by VietLV (Modified date 01/06/2011)
            }
            lf = HttpUtility.HtmlDecode(ConvertNewlinesToSingleSpaces(StripHTML(lf)));
            cf = HttpUtility.HtmlDecode(ConvertNewlinesToSingleSpaces(cf));
            // Remote summary in content if conten include summary (Viet lv 21/09/2011)
            if (cf.Contains(lf))
            {
                cf = cf.Replace(lf, "");
            }
            cf = cf.Trim();
            #endregion

            #region @ManualTitle varchar(200)
            string ManualTitle = "";
            ManualTitle = tf;
            #endregion

            #region MetaTitle nvarchar(200)
            string MetaTitle = tf;

            #endregion

            #region @MetaDescription nvarchar(400)
            string MetaDescription = "";
            MetaDescription = lf;

            #endregion


           #region Insert Link Image DB
              ///Viet bỏ rào ngày 01/09/2011
              ///Lấy ảnh trong phần bài viết để lưu vào trong thư mục
            var _img = new ImgDes();
            IList<string> l = new List<string>();
            if (!string.IsNullOrEmpty(cf))
            {
                if (_w.TheUFilter.ConFilter.ImagePathFilter != null)
                {
                    if (_w.TheUFilter.ConFilter.ImagePathFilter.TypeImage.Equals(PathImageFilter.GetText(SDT.CrawlSystem.Servers.Filters.PathImageFilter.ImageTypes.DOMAIN)))
                        _img.BaseURL = _w.TheUFilter.Prefix;
                    else
                        _img.BaseURL = _w.Url;
                }
                else
                    _img.BaseURL = _w.TheUFilter.Prefix;
                var sourceHtml = "";
                if (cf.Trim() != null && cf.Trim().Length > 0 && (cf.Contains("img") || cf.Contains("IMG")))
                {
                    sourceHtml = cf;
                }                
                else sourceHtml = hf;
                string allowtag="p|u|img|embed|small|em|strong";
                sourceHtml = sourceHtml.Trim().Replace("IMG", "img");
                sourceHtml = StripTagsAndAttributes(sourceHtml.Replace("  ",""), allowtag);
                l = _img.GetUrlAttributeSrc(_img.BaseURL, sourceHtml);  // Viet 21/09/2011 sua tham so dau vao: mac dinh: (_img.BaseURL, contentRawHtml) =>  (_img.BaseURL, hf);
                if (l != null && l.Count > 0)
                {
                    int countImage = 0;
                    foreach (var img in l)
                    {
                        if (countImage == 0)
                            svDownload.Insert_LinkDownload(_w.Uid, l[0], (byte)Global.DownloadStatus.NEW, (byte)Global.FileType.IMG, true, _w.PageId, _w.ChannelId, l.Count);
                        countImage++;
                    }
                }
                else
                {
                    new LogException().Write(string.Format("  Module {0} --Msg {1} ", "lỗi ảnh", _w.Uid + "--" + _w.Url));
                }
                IList<string> ltemp = _img.GetUrlAttributeSrc(cf);
                cf = svNews.SetContentHaveAbsIMG(ltemp, l, cf);
            }
           
           #endregion

            #region Create FTS

            //var b = new StringBuilder();
            //b.AppendLine(tf).Append(lf).AppendLine(cf);
            //var FTS = StringConvertor.ToFTS(b.ToString());
            #endregion

            #region Insert news into database
           
            int id = 0;
            if (tf.Equals(string.Empty) || lf.Equals(string.Empty) || cf.Equals(string.Empty) || df >= DateTime.Now)
            {
                #region checkError
                bool errorTitle = false;
                bool errorLead = false;
                bool errorContent = false;
                bool errorDateTime = false;
                if (tf.Equals(string.Empty))
                    errorTitle = true;
                if (lf.Equals(string.Empty))
                    errorLead = true;
                if (cf.Equals(string.Empty))
                    errorContent = true;
                if (df >= DateTime.Now)
                    errorDateTime = true;
                #endregion
                id = svErrorNews.WC_ErrorNewsInsert(_w.ChannelId, _w.Uid, _w.PageId, df, DateTime.Now, tf, lf, cf, _w.Url, string.Empty, (byte)Global.NewsStatus.NEW, errorTitle, errorLead, errorContent, errorDateTime);
                return true;
            }
            if (!tf.Equals(string.Empty) && !lf.Equals(string.Empty) && !cf.Equals(string.Empty) && df < DateTime.Now)
            {
                //Ảnh đại diện không chèn vào ở đây mà đặt trong store
                // Insert to Other Database 
                // Created by Vietlv
                // Created date 04/07/2011;
                SystemSpiderDataContext eSpiderDataContext = new SystemSpiderDataContext();
                var source = eSpiderDataContext.Spider_Sites.Where(e => e.SiteId == _w.ChannelId).Select( e => e.Name).SingleOrDefault();
                id = svNews.InsertArticleToOtherDatabase(_w.ChannelId, _w.Uid, _w.PageId, (int)Global.NewsStatus.NEW, df, tf, lf, cf, string.Empty, true, _w.Url, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, (byte)Global.NewsStatus.NEW, source.ToString());
                
                if (id > 0)
                {
                    _w.ChangeStatus(UrlLoader.Status.SUCCESS);
                }
                
                // id = svNews.WC_Insert(_w.ChannelId, _w.Uid, _w.PageId, (int)Global.NewsStatus.NEW, df, tf, lf, cf, string.Empty, true, _w.Url,string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, (byte)Global.NewsStatus.NEW);
            }
            #endregion

            //if (id > 0)
            //{
            //    #region Save File
            //    SaveFiles saveFiles = new SaveFiles(Global.Instance.HtmlFolder, id, _w.ChannelId, _w.PageId, _w.Uid, _w.RawHtml, hf, txt, tf, df, lf, cf, _w.TheUFilter.Enc);
            //    saveFiles.SaveFileTask();
            //    #endregion
                              
            //}
            
        #endregion



            _w.ClearResult();

            return true;
        }



        public override WorkResultHandlerDelegate ResultHandler
        {
            protected get
            {
                if (base.ResultHandler == null)
                    ResultHandler = new WorkResultHandlerDelegate(SaveResult);
                return base.ResultHandler;
            }
            set { base.ResultHandler = value; }
        }

    }
}
